/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

syntax = "proto2";

package tflite.evaluation;

// Defines the functionality executed by an EvaluationStage.
//
// Next ID: 3
message ProcessSpecification {
  oneof params {
    ImagePreprocessingParams image_preprocessing_params = 1;
    TopkAccuracyEvalParams topk_accuracy_eval_params = 2;
  }
}

// Latency numbers in microseconds, based on all EvaluationStage::Run() calls so
// far.
//
// Next ID: 6
message LatencyMetrics {
  // Latency for the last Run.
  optional int64 last_us = 1;
  // Maximum latency observed for any Run.
  optional int64 max_us = 2;
  // Minimum latency observed for any Run.
  optional int64 min_us = 3;
  // Sum of all Run latencies.
  optional int64 sum_us = 4;
  // Average latency across all Runs.
  optional double avg_us = 5;
}

// Contains process-specific metrics, which may differ based on what an
// EvaluationStage does.
//
// Next ID: 3
message ProcessMetrics {
  optional LatencyMetrics total_latency = 1;

  oneof stage_metrics {
    TopkAccuracyEvalMetrics topk_accuracy_metrics = 2;
  }
}

// Parameters that define how images are preprocessed.
//
// Next ID: 5
message ImagePreprocessingParams {
  // Required.
  optional int32 image_height = 1;
  // Required.
  optional int32 image_width = 2;
  // Same as tflite::TfLiteType.
  optional int32 output_type = 3;
  // Fraction for central-cropping.
  // A central cropping-fraction of 0.875 is considered best for Inception
  // models, hence the default value. See:
  // https://github.com/tensorflow/tpu/blob/master/models/experimental/inception/inception_preprocessing.py#L296
  // Set to 0 to disable cropping.
  optional float cropping_fraction = 4 [default = 0.875];
}

// Parameters that define how top-K accuracy is evaluated.
//
// Next ID: 2
message TopkAccuracyEvalParams {
  // Required.
  optional int32 k = 1;
}

// Metrics from top-K accuracy evaluation.
//
// Next ID: 2
message TopkAccuracyEvalMetrics {
  // A repeated field of size |k| where the ith element denotes the fraction of
  // samples for which the correct label was present in the top (i + 1) model
  // outputs.
  // For example, topk_accuracy_percentages(1) will contain the fraction of
  // samples for which the model returned the correct label as the top first or
  // second output.
  repeated float topk_accuracy_percentages = 1;
}
